# -*- coding: utf-8 -*-

BOT_NAME = 'spider'
COOKIES_ENABLED = False
SPIDER_MODULES = ['spiders']
NEWSPIDER_MODULE = 'spiders'

ROBOTSTXT_OBEY = False

# change cookie
DEFAULT_REQUEST_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36',
    'Cookie': '_T_WM=1da2e9ca8d3b1bb89efa62d7e63debea; SCF=AuhN5LgrCBmbrbLoe9_CbJ6fLz-xgdvMVPz0EjnjEXRwYINTH_l3U1PhZjk3j_XGoDhngOabil1F7C55Nj4T6QU.; SUB=_2A25N5QcdDeRhGeVH7FsW9CnMyjyIHXVvKalVrDV6PUJbktB-LU_skW1NTzzlyET6GbDW-Ytx4GGvb6PW7BZeODE6; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W59fzWhkvnK9Z3p9hUuCgPK5NHD95Q01KM4S0BNeh27Ws4Dqcj-Sh5XdJvX9JvfdJBLxK-L1KMLB-qt; SSOLoginState=1625388877'}

CONCURRENT_REQUESTS = 16

DOWNLOAD_DELAY = 3

DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': None,
    'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': None,
    'middlewares.IPProxyMiddleware': 100,
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 101,
}

ITEM_PIPELINES = {
    'pipelines.MongoDBPipeline': 300,
}

MONGO_HOST = '127.0.0.1'
MONGO_PORT = 27017
